package com.terren.spider.core.html.downloader;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.annotation.ThreadSafe;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.utils.URIUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.terren.spider.core.html.configuration.Configuration;
import com.terren.spider.core.html.configuration.DynamicType;
import com.terren.spider.entity.config.SysConfig;
import com.terren.spider.entity.core.ProxyRequestParam;
import com.terren.spider.util.common.Lookup3Hash;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.AbstractDownloader;
import us.codecraft.webmagic.downloader.HttpClientGenerator;
import us.codecraft.webmagic.downloader.HttpClientRequestContext;
import us.codecraft.webmagic.downloader.HttpUriRequestConverter;
import us.codecraft.webmagic.model.HttpRequestBody;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.ProxyProvider;
import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.utils.CharsetUtils;
import us.codecraft.webmagic.utils.HttpClientUtils;
import us.codecraft.webmagic.utils.HttpConstant;

@ThreadSafe
public abstract class BasicHttpClientDownloader extends AbstractDownloader {
	protected Logger logger = LoggerFactory.getLogger(getClass());

	protected final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();

	protected HttpClientGenerator httpClientGenerator = new HttpClientGenerator();

	protected HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter();

	protected ProxyProvider proxyProvider;

	protected boolean responseHeader = true;

	protected Configuration configuration;
	

	public BasicHttpClientDownloader() {
		super();
	}

	public BasicHttpClientDownloader(Configuration configuration) {
		super();
		this.configuration = configuration;
	}

	public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) {
		this.httpUriRequestConverter = httpUriRequestConverter;
	}

	public void setProxyProvider(ProxyProvider proxyProvider) {
		this.proxyProvider = proxyProvider;
	}

	private CloseableHttpClient getHttpClient(Site site) {
		if (site == null) {
			return httpClientGenerator.getClient(null);
		}
		String domain = site.getDomain();
		CloseableHttpClient httpClient = httpClients.get(domain);
		if (httpClient == null) {
			synchronized (this) {
				httpClient = httpClients.get(domain);
				if (httpClient == null) {
					httpClient = httpClientGenerator.getClient(site);
					httpClients.put(domain, httpClient);
				}
			}
		}
		return httpClient;
	}

	@Override
	public Page download(Request request, Task task) {
		if (task == null || task.getSite() == null) {
			throw new NullPointerException("task or site can not be null");
		}
		if (ifHttpProxy()) {
			// init request
			logger.info("use http proxy:"+request.toString());
			initHttpProxyRequest(request);
		}
		CloseableHttpResponse httpResponse = null;
		CloseableHttpClient httpClient = getHttpClient(task.getSite());
		Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(task) : null;
		HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy);
		Page page = Page.fail();
		try {
			httpResponse = httpClient.execute(requestContext.getHttpUriRequest(),
					requestContext.getHttpClientContext());
			int retryTimes = 0;//重试次数
			while(needRetry(httpResponse)) {
				retryTimes++;
				if (retryTimes>configuration.getRetryTimes()) {//大于3次 退出
					page.setSkip(true);
					logger.warn("download page {} error, page skip", request.getUrl());
					return page;
				}
				Thread.sleep(3*1000);
				logger.info("[retry times:"+retryTimes+"]");
				httpResponse = httpClient.execute(requestContext.getHttpUriRequest(),
						requestContext.getHttpClientContext());
			}
			page = handleResponse(requestContext, request,
					request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse,
					task);
			onSuccess(request);
			logger.info("downloading page success {}", request.getUrl());
			return page;
		} catch (IOException e) {
			logger.warn("download page {} error", request.getUrl(), e);
			onError(request);
			return page;
		} catch (Exception e) {
			logger.warn("download page {} error", request.getUrl(), e);
			onError(request);
			return page;
		} 
		finally {
			if (httpResponse != null) {
				// ensure the connection is released back to pool
				EntityUtils.consumeQuietly(httpResponse.getEntity());
			}
			if (proxyProvider != null && proxy != null) {
				proxyProvider.returnProxy(proxy, page, task);
			}
		}
	}

	@Override
	public void setThread(int thread) {
		httpClientGenerator.setPoolSize(thread);
	}

	protected Page handleResponse(HttpClientRequestContext requestContext, Request request, String charset,
			HttpResponse httpResponse, Task task) throws IOException {
		byte[] bytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
		String contentType = httpResponse.getEntity().getContentType() == null ? ""
				: httpResponse.getEntity().getContentType().getValue();
		Page page = new Page();
		page.setBytes(bytes);
		if (!request.isBinaryContent()) {
			if (charset == null) {
				charset = getHtmlCharset(contentType, bytes);
			}
			page.setCharset(charset);
			page.setRawText(new String(bytes, charset));
		}

		page.setUrl(new PlainText(getRealTargetUrl(request)));
		page.setRequest(request);
		page.setStatusCode(httpResponse.getStatusLine().getStatusCode());
		page.setDownloadSuccess(true);
		if (responseHeader) {
			page.setHeaders(HttpClientUtils.convertHeaders(httpResponse.getAllHeaders()));
		}
		if (null != configuration && configuration.getProxyRequestParam().getContentUseProxy()
				&& configuration.getProxyRequestParam().getEntryUseProxy()) {// 使用代理

			page.getRequest().putExtra(SysConfig.LAST_REDIRECT_URL,
					httpResponse.getFirstHeader("X-Real-Url") != null
							? httpResponse.getFirstHeader("X-Real-Url").getValue()
							: null);// 最终跳转的页面地址
		} else {// 不使用代理
			page.getRequest().putExtra(SysConfig.LAST_REDIRECT_URL, getFinalHTTPLocation(requestContext));// 最终跳转的页面地址
		}

		return page;
	}

	protected String getHtmlCharset(String contentType, byte[] contentBytes) throws IOException {
		String charset = CharsetUtils.detectCharset(contentType, contentBytes);
		if (charset == null) {
			charset = Charset.defaultCharset().name();
			logger.warn("Charset autodetect failed, use {} as charset. Please specify charset in Site.setCharset()",
					Charset.defaultCharset());
		}
		return charset;
	}

	/**
	 * 获取最后1次跳转的url
	 * 
	 * @param requestContext
	 * @return
	 */
	protected String getFinalHTTPLocation(HttpClientRequestContext requestContext) {
		String last_redirect_url = null;
		try {
			HttpHost target = requestContext.getHttpClientContext().getTargetHost();
			List<URI> redirectLocations = requestContext.getHttpClientContext().getRedirectLocations();
			URI location = URIUtils.resolve(requestContext.getHttpUriRequest().getURI(), target, redirectLocations);
			logger.debug("Final HTTP location: " + location.toASCIIString());
			last_redirect_url = location.toASCIIString();
		} catch (URISyntaxException e) {
			logger.error("getFinalHTTPLocation error", e);
		}
		return last_redirect_url;
	}

	/**
	 * 是否使用HTTP代理
	 * 
	 * @return
	 */
	protected boolean ifHttpProxy() {
		return configuration != null && configuration.getProxyRequestParam().getEntryUseProxy()
				&& configuration.getProxyRequestParam().getContentUseProxy();
	}

	/**
	 * 最终请求的url
	 * 
	 * @param request
	 * @return
	 */
	protected String getRealTargetUrl(Request request) {
		String url = request.getUrl();
		if (ifHttpProxy()) {
			url = request.getExtra("url") != null ? (String) request.getExtra("url") : formatProxyUrl(url);
		}
		return url;
	}

	protected String formatProxyUrl(String url) {
		if (null != url) {
			String entryUrl = configuration.getEntry().getEntryUrl();
			String domain = configuration.getDomain();
			if(entryUrl==null||entryUrl.equals("")) {
				return url;
			}
			ProxyRequestParam param = configuration.getProxyRequestParam();
			String host = param.getProxyHost();
			Integer port = param.getProxyPort();
			/*
			 * Pattern patternForProtocal = Pattern.compile("://.+"); String prex =
			 * patternForProtocal.matcher(url).replaceAll("");// http or https String
			 * replace_host = prex + "://" + configuration.getDomain() + "/";
			 */
			
			String replace_host = null;
			String act = configuration.getEntry().getAct();
			switch (act) {
			case "baiduNews":
				replace_host = StringUtils.substringBefore(entryUrl, "?");
				break;
			case "baiduSearch":
				replace_host = StringUtils.substringBefore(entryUrl, "?");
				break;

			default:
				String http = StringUtils.substringBefore(entryUrl, "://");
				replace_host = http + "://"+domain+ "/";
				break;
			}
			
			// http://192.168.20.8:13003/
			String proxyServer = "http://" + host + ":" + port + "/";
			String realTarget = "";
			if(StringUtils.contains(url, proxyServer)) {
				realTarget = StringUtils.replace(url, proxyServer, replace_host);
			} else {
				if(!StringUtils.contains(url, replace_host)
						&&!StringUtils.startsWith(url, "http")) {
					realTarget = replace_host + url;
				}
				else {
					realTarget = url;
				}
			}
			return realTarget;
		}
		return null;
	}

	protected void initHttpProxyRequest(Request request) {
		if (ifHttpProxy()) {
			String realTarget = getRealTargetUrl(request);
			ProxyRequestParam param = configuration.getProxyRequestParam();
			String host = param.getProxyHost();
			Integer port = param.getProxyPort();
			DynamicType dynamicType = param.getUrlDynamic();

			Map<String, Object> params = new HashMap<>();
			params.put("url", realTarget);
			request.putExtra("url", realTarget);
			params.put("dynamic", dynamicType.isDynamic());
			request.putExtra("dynamic", dynamicType.isDynamic());
			if (null != param.getPage()) {
				params.put("page", param.getPage());
				request.putExtra("page", param.getPage());
			}
			if (null != param.getTimeout()) {
				params.put("timeout", param.getTimeout());
				request.putExtra("timeout", param.getTimeout());
			}
			if (null != param.getTimewait()) {
				params.put("timewait", param.getTimewait());
				request.putExtra("timewait", param.getTimewait());
			}
			if (param.getUseAbuyun()) {//使用阿布云代理
				params.put("aby", param.getUseAbuyun());
				request.putExtra("aby", param.getUseAbuyun());
			}

			request.setRequestBody(HttpRequestBody.form(params, "utf-8"));
			request.setMethod(HttpConstant.Method.POST);
			String targetUrl = "http://" + host + ":" + port + "/?urlid=" + Lookup3Hash.toID(request.getUrl());// 加urlid是为了去掉重复提交的url
			request.setUrl(targetUrl);
		}
	}

	/**
	 * 是否需要重试连接
	 * @param httpResponse
	 * @return
	 */
	protected boolean needRetry(CloseableHttpResponse httpResponse) {
		boolean flag = false;
		Integer status = httpResponse.getStatusLine().getStatusCode();
		switch (status) {
		// --- 5xx Server Error ---
		case HttpStatus.SC_INTERNAL_SERVER_ERROR://500
			flag = true;
			logger.info("# internal server error [status code:"+status+"]");
			break;
		case HttpStatus.SC_NOT_IMPLEMENTED://501
			flag = true;
			logger.info("# Not Implemented [status code:"+status+"]");
			break;
		case HttpStatus.SC_BAD_GATEWAY://502
			flag = true;
			logger.info("# 502 Bad Gateway [status code:"+status+"]");
			break;
		case HttpStatus.SC_GATEWAY_TIMEOUT://504
			flag = true;
			logger.info("# gateway timeout [status code:"+status+"]");
			break;
		case HttpStatus.SC_SERVICE_UNAVAILABLE://503
			flag = true;
			logger.info("# Service Unavailable [status code:"+status+"]");
			break;
		case HttpStatus.SC_HTTP_VERSION_NOT_SUPPORTED://505
			flag = true;
			logger.info("# HTTP Version Not Supported [status code:"+status+"]");
			break;
		case HttpStatus.SC_INSUFFICIENT_STORAGE://507
			flag = true;
			logger.info("# Insufficient Storage [status code:"+status+"]");
			break;

		default:
			flag = false;
			break;
		}
		return flag;
	}
	
	
	protected void onSuccess(Request request) {
		
	}

	protected void onError(Request request) {
	}

}
